import json
import time


# 读取JSON文件
def read_json_file(file_path):
    with open(file_path, 'r', encoding='utf-8') as file:
        data = json.load(file)
        return data


def read_json_file_two(file_path):
    with open(file_path, 'r') as file:
        data = json.load(file)
        return data


class VideoBean:
    id = 0
    name = ''
    url = ''
    e = ''
    j = ''
    tags = ''
    pUrl = ''
    vUrl = ''

    def __eq__(self, other):
        return self.pUrl == other.pUrl and self.vUrl == other.vUrl

    def __hash__(self):
        return hash(self.pUrl + self.vUrl)


class SpiderInfo:
    # 爬虫的名称
    name = ''
    # 网址
    url = ''
    # 爬取下来的数据是否需要合并
    isMerge = False
    # 是否需要验证videoUrl的有效性
    isPlay = False

    def __init__(self, name, url, isMerge, isPlay):
        # 爬虫的名称
        self.name = name
        # 网址
        self.url = url
        # 爬取下来的数据是否需要合并
        self.isMerge = isMerge
        # 是否需要验证videoUrl的有效性
        self.isPlay = isPlay

def split_array_with_remainder(arr, parts=5):
    n = len(arr)
    size = n // parts
    result = [arr[i*size : (i+1)*size] for i in range(parts-1)]
    result.append(arr[(parts-1)*size:])  # 添加剩余部分
    return result
# 1完整数据并去重复
if __name__ == '__main__':
    print('开始')
    timestamp = int(time.time())
    jsonStr = read_json_file('D:\\ya.json')
    print("原始数据一共: " + str(len(jsonStr)))
    result = split_array_with_remainder(jsonStr, 2)
    for i in range(len(result)):
        # 将JSON数组转换为字符串
        json_string = json.dumps(result[i], ensure_ascii=False)
        # 将字符串写入txt文件
        with open('D:\\ya'+str(i+1)+'.json', 'w', encoding='utf-8') as file:
            file.write(json_string)



    # for i in jsonStr:
    #     video = VideoBean()
    #     video.id = 0
    #     video.name = i['name']
    #     video.url = i['url']
    #     video.tags = i['tags']
    #     video.pUrl = i['pUrl']
    #     video.vUrl = i['vUrl']
    #     videoList.append(video)
    endTime = int(time.time())
    print(str(endTime - timestamp))
    print('结束')
